{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF-IDF算法关键词提取的结果:\n",
      "会徽"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import jieba\n",
    "import jieba.posseg as psg\n",
    "def load_data(pos=False,corpus_path='corpus.txt'):\n",
    "    doc_list=[]\n",
    "    for line in open(corpus_path,'r',encoding='utf-8'):\n",
    "        content=line.strip()\n",
    "        cut_list=cutWord(content,pos)\n",
    "        filter_list=removeWord(cut_list,pos)\n",
    "        doc_list.append(filter_list)\n",
    "    return doc_list\n",
    "def cutWord(sentence,pos=False):\n",
    "    if not pos:\n",
    "        cut_list=jieba.cut(sentence)\n",
    "    else:\n",
    "        cut_list=psg.cut(sentence)\n",
    "    return cut_list\n",
    "def removeWord(seg_list,pos=False):\n",
    "    stop_word_path='stops_list.txt'\n",
    "    stopword_list=[sw.replace('\\n', '') for sw in open(stop_word_path,'r',encoding='utf-8').readlines()]\n",
    "    filter_list=[]\n",
    "    for seg in seg_list:\n",
    "        if not pos:\n",
    "            word=seg\n",
    "            flag='n'\n",
    "        else:\n",
    "                word=seg.word\n",
    "                flag=seg.flag\n",
    "                if not flag.startswith('n'):\n",
    "                    continue\n",
    "                if not word in stopword_list and len(word)>1:\n",
    "                    filter_list.append(word)\n",
    "        return filter_list\n",
    "def get_tf(word_list):\n",
    "    tf_dic={}\n",
    "    for word in word_list:\n",
    "        tf_dic[word]=tf_dic.get(word,0.0)+1.0\n",
    "    tt_count=len(word_list)\n",
    "    for k,v in tf_dic.items():\n",
    "        tf_dic[k]=float(v)/tt_count\n",
    "    return tf_dic\n",
    "def get_idf(doc_list):\n",
    "    idf_dic={}\n",
    "    tt_count=len(doc_list)\n",
    "    for doc in doc_list:\n",
    "        for word in set(doc):\n",
    "            idf_dic[word]=idf_dic.get(word,0.0)+1.0\n",
    "    for k,v in idf_dic.items():\n",
    "        idf_dic[k]=math.log(tt_count/(1.0+v))\n",
    "    default_idf=math.log(tt_count/(1.0))\n",
    "    return idf_dic,default_idf\n",
    "def get_tfidf(idf_dic,default_idf,word_list,keyword_num):\n",
    "    tfidf_dic={}\n",
    "    for word in word_list:\n",
    "        idf=idf_dic.get(word,default_idf)\n",
    "        tf_dic=get_tf(word_list)\n",
    "        tf=tf_dic.get(word,0)\n",
    "        tfidf=tf*idf\n",
    "        tfidf_dic[word]=tfidf\n",
    "    for k,v in sorted(tfidf_dic.items(),key=lambda x:x[1],reverse=True)[:keyword_num]:\n",
    "        print(k+\"\",end='')\n",
    "def tfidf_extract(word_list,pos=False,keyword_num=10):\n",
    "    doc_list=load_data(pos)\n",
    "    idf_dic,default_idf=get_idf(doc_list)\n",
    "    tfidf_model=get_tfidf(idf_dic,default_idf,word_list,keyword_num)\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "\n",
    "    text='冬奥会会徽以汉字“冬”为灵感来源，运用中国书法的艺术形态， 将厚重的东方文化底蕴与国际化的现代风格融为一体，呈现出新时代的中国新形象、新梦想，传递出新时代中国为办好北京冬奥会，圆冬奥之梦，实现“三亿人参与冰雪运动”目标，圆体育强国之梦，推动世界冰雪运动发展，为国际奥林匹克运动做出新贡献的不懈努力和美好追求。会徽图形上半部分展现滑冰运动员的造型，下半部分表现滑雪运动员的英姿。中间舞动的线条流畅且充满韵律，代表举办地起伏的山峦、赛场、冰雪滑道和节日飘舞的丝带，为会徽增添了节日喜庆的视觉感受，也象征着北京冬奥会将在中国春节期间举行。会徽以蓝色为主色调，寓意梦想与未来，以及冰雪的明亮纯洁。红黄两色源自中国国旗，代表运动的激情、青春与活力。在“BEIJING 2022”字体的形态上汲取了中国书法与剪纸的特点，增强了字体的文化内涵和表现力，也体现了与会徽图形的整体感和统一性。'\n",
    "    cut_list=cutWord(text,pos=True)\n",
    "    filter_list=removeWord(cut_list,pos=True)\n",
    "    print('TF-IDF算法关键词提取的结果:')\n",
    "    tfidf_extract(filter_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
